#!/usr/bin/env python
#-*- coding:utf-8 -*-
import os,sys
import re
import urllib
import random
from wwpy.util.Time import Time
from wwpy.util.Logger import logger
from wwpy.db.DBI import DBI,DBC

class NotImplement(Exception):
    pass

class LineProcessor(object):
    """
    行处理器，基类
    """
    def __init__(self,line=None,matcher=None,field_dict=None,field_replacer=None,**kwargs):
        self.line=line
        self.matcher=matcher
        self.field_dict=field_dict
        self.field_replacer=field_replacer
        self.time=Time()
        self.fields={}
    
    def define_line(self):
        """define a line,can omit."""
        pass
        
    def define_matcher(self):
        """define a line mather,must override."""
        raise NotImplement
        
    def define_field_dict(self):
        """
        define a field dict,like field_dict={field_name:(index,default)},must override.
        """
        raise NotImplement
        
    def define_field_replacer(self):
        """
        define a field replacer,this can rename the field,can omit.
        """
        pass
    
    def init_field_dict(self):
        """
        make field_dict={field:(index,default)}
        """
        if not isinstance(self.field_dict,dict):
            raise Exception('field dict must be dict type.')
        for field,value in self.field_dict.items():
            if isinstance(value,(tuple,list)) and len(value)==2:
                continue
            self.field_dict[field]=(value,'')
        
    def decode_str(self,encoding_str,decoding='utf-8'):
        """
        decode str,str like http%3A%2F%2Ftj.58.com%2F,
        and switch it into 'http://tj.58.com/'
        """
        decode_str=urllib.unquote(encoding_str)
        if isinstance(decode_str,str):
            return decode_str
        return decode_str.decode(decoding)
        
    def process_line(self,line,matcher=None,default_fields=None):
        """
        process a line,and return {field:value}
        default_fields=[(field,default)] or default_fields=[field]
        """
        if matcher is not None:
            self.matcher=matcher
        if self.matcher is None:
            return {'line':line}
        try:
            line_fields={}
            default_fields=map(lambda f: f if isinstance(f,(tuple,list)) and len(f)==2 else (f,''),default_fields)
            field_names=tuple(map(lambda f:f[0],default_fields))
            m=re.match(self.matcher,line)
            if m:
                for field,value in default_fields:
                    line_fields[field]=m.group(field) or value
                return line_fields
            return None
        except Exception,e:
            logger.error("Process line error:"+str(e))
            return None
        
    def process_gif(self,gif):
        """
        process gif params,switch params into {key:value}
        gif=line_fields['gif']
        """
        if not gif:
            return None
        try:
            fields=gif.split('&')
            gif_fields={}
            for field in fields:
                if not field:
                    continue
                if field.find('=')==-1:
                    continue
                key,value=field.split('=',1)
                if key not in gif_fields:
                    gif_fields[key]=value
            return gif_fields
        except Exception,e:
            logger.error("Process gif error:"+str(e))
            return None
    
    def process_url(self,url,default_fields=None,is_decoded=False):
        """
        url like http://t.58.com/bj/68784548064773007/?linkid=bj_liebiao_xxyl_2
        process url,make it into {host:value,uri:value:params:value}
        """
        if not url:
            return None
        if default_fields is None:
            default_fields=[('host',''),('uri','/'),('params','')]
        else:
            default_fields=map(lambda f:f if isinstance(f,(tuple,list)) and len(f)==2 else (f,''),default_fields)
        field_names=tuple(map(lambda f:f[0],default_fields))
        try:
            url_fields={}
            if is_decoded:
                url=self.decode_str(url)
            m=re.match(r'^(?:http(?:s)?://)?(?P<%s>[^\/]+)(?P<%s>[^\?]+)?(?:\?(?P<%s>.*))?$' % field_names,url)
            if not m:
                return None
            for field,value in default_fields:
                url_fields[field]=m.group(field) or value
            return url_fields
        except Exception,e:
            logger.error("Process url error:"+str(e))
            return None
        
    def process_time(self,timestamp,field_dict=None):
        """
        process timestamp,and switch into {time:value}
        field_dict={field:fmt}
        """
        if not timestamp:
            timestamp=Time.timestamp()
        if field_dict is None:
            field_dict={
                'time':'%Y-%m-%d %X',
                'date':'%Y%m%d',
                'year':'%Y',
                'month':'%M',
                'day':'%d',
                'hour':'%H',
            }
        try:
            time_fields={}
            time_fields['timestamp']=timestamp
            for field,fmt in field_dict.items():
                Time.time_fmt=fmt
                time_fields[field]=Time.time2str(timestamp)
            return time_fields
        except Exception,e:
            logger.error("Process time error:"+str(e))
            return None
        
    def process_other(self,other_func=None,fields=None):
        if other_func is None:
            return None
        try:
            other_fields=other_func(fields)
            return other_fields
        except Exception,e:
            logger.error("Process other error:"+str(e))
            return None
        
    def get_fields(self,line,to_list=True):
        """
        make a original line into aim fields,must override.
        """
        raise NotImplement
        
class GaLineProcessor(LineProcessor):
    """
    LineProcessor for ga access log.
    """
    def __init__(self,**kwargs):
        super(GaLineProcessor,self).__init__(**kwargs)
        self.line=self.define_line()
        self.matcher=self.define_matcher(gif=kwargs.get('gif',None))
        self.field_dict=self.define_field_dict()
        self.field_replacer=self.define_field_replacer()
        self.init_field_dict()
        
    def define_matcher(self,gif=None):
        if gif is None:
            gif='/log/caipiao/collectlog\.gif'
        matcher=r'(?P<ip>[\d\.]{7,})\s.*\[(?P<nginx_time>[^\s]+)\s(?:[^\s\]]+)\s?(?P<timestamp>[^\]]+)?\]\s.*GET.*'+gif+'\?(?P<gif>[^\s]+)(?:.+?)(?P<timestamp2>[\d\.]{14,})?$'
        return matcher
    
    def define_field_dict(self):
        return {
            'session_id':0,
            'visitor_id':1,
            'first_visit_time':2,
            'last_visit_time':3,
            'current_visit_time':4,
            'visit_count':5,
            'click_timestamp':6,
            'click_time':7,
            'click_date':8,
            'click_year':9,
            'click_month':10,
            'click_day':11,
            'click_hour':12,
            'ip':13,
            'host':14,
            'uri':(15,'/'),
            'params':16,
            'refer_host':17,
            'refer_uri':18,
            'refer_params':19,
            'utm_source':20,
            'utm_campaign':21,
            'utm_medium':22,
            'utm_term':23,
            'utm_content':24,
            'request_type':25,
            'request_extend':26,
        }

    def define_field_replacer(self):
        return {
            'autmcsr':'utm_source',
            'autmccn':'utm_campaign',
            'autmcmd':'utm_medium',
            'autmctr':'utm_term',
            'autmcct':'utm_content',
            'autmt':'request_type',
            'autme':'request_extend',
            'timestamp':'click_timestamp',
            'time':'click_time',
            'date':'click_date',
            'year':'click_year',
            'month':'click_month',
            'day':'click_day',
            'hour':'click_hour',
        }
        
    def define_line(self):
        return '''123.150.182.169 - - [15/Jan/2013:00:03:28 +0800 1358179408.563] "GET /log/caipiao/collectlog.gif?&autmn=1618366689&autmhn=caipiao.58.com&autmcs=UTF-8&autmsr=800x600&autmsc=24-bit&autmul=zh-cn&autmje=0&autmfl=10.0%20r22&autmhid=629286436&autmr=http%3A%2F%2Ftj.58.com%2F&autmp=%2F%3Ffrom%3Dhome-hotkey-caipiao&autmcc=__autma%3D253535702.400542011.1358179409.1358179409.1358179409.1%3B%2B__autmz%3D253535702.1358179409.1.1.autmcsr%3Dtj.58.com%7Cautmccn%3D(referral)%7Cautmcmd%3Dreferral%7Cautmcct%3D%2F%3B&id58=05dvaFD0K74EiQ9ayeIbAg==&refurl=http://tj.58.com/?&test=abc HTTP/1.1" 200 0 "http://caipiao.58.com/?from=home-hotkey-caipiao" "-" "-" 1358179408.863'''
        
    def process_line(self,line,default_fields=None):
        if self.matcher is None:
            return {'line':line}
        try:
            line_fields={}
            m=re.match(self.matcher,line)
            if m:
                line_fields['ip']=m.group('ip') or ''
                timestamp=m.group('timestamp') or m.group('timestamp2') or self.time.time2str(self.time.nginx_time(m.group('nginx_time'))) or self.time.timestamp()
                line_fields['timestamp']=timestamp
                line_fields['gif']=m.group('gif') or ''
                return line_fields
            return None
        except Exception,e:
            logger.error("Process line error:"+str(e))
            return None
        
    def process_gif(self,gif):
        """
        gif=line_fields['gif']
        """
        gif_fields=super(GaLineProcessor,self).process_gif(gif)
        if not gif_fields:
            return None
        try:
            for key in ['autmcc','autmhn','autmp','autmr']:
                if not gif_fields.has_key(key):
                    gif_fields[key]=''
            return gif_fields
        except Exception,e:
            logger.error("Process gif error:"+str(e))
            return None
        
    def process_cookie(self,cookie,timestamp=None):
        """
        cookie=gif_fields['autmcc']
        """
        if not cookie:
            return {
                '__autma':'',
                '__autmz':'',
            }
        try:
            cookie_fields={}
            cookie=self.decode_str(cookie)
            cookie=cookie.replace(';+',';')
            fields=cookie.split(';')[:-1]
            for field in fields:
                key,value=field.split('=',1)
                if not cookie_fields.has_key(key):
                    cookie_fields[key]=value
            for key in ['__autma','__autmz']:
                if not cookie_fields.has_key(key):
                    cookie_fields[key]=''
            return cookie_fields
        except Exception,e:
            self.logger.error("Process cookie error:"+str(e))
            return None
    
    def process_visit(self,visit,timestamp=None):
        """
        visit=cookie_fields['__autma']
        __autma:area_hash.visitor_id.first_visit_time.last_visit_time.current_visit_time.visit_count;
        """
        if not timestamp:
            timestamp=self.time.timestamp()
        try:
            visit_fields={}
            if not visit:
                visit_time=self.time.time2str(timestamp) or self.time.now()
                visit_fields['visitor_id']=''.join(map(lambda i:random.randint(0,10),range(8)))+str(timestamp).split('.')[0][4:]
                visit_fields['visit_count']='1'
                visit_fields['session_id']=visit_fields['visitor_id']+visit_fields['visit_count']
                visit_fields['first_visit_time']=visit_time
                visit_fields['last_visit_time']=visit_time
                visit_fields['current_visit_time']=visit_time
                return visit_fields
            area_hash,visitor,fvt,lvt,cvt,visit_count=visit.split('.')
            visit_fields['visitor_id']=visitor+fvt[4:]
            visit_fields['session_id']=visit_fields['visitor_id']+visit_count
            visit_fields['visit_count']=visit_count
            visit_fields['first_visit_time']=self.time.time2str(fvt)
            visit_fields['last_visit_time']=self.time.time2str(lvt)
            visit_fields['current_visit_time']=self.time.time2str(cvt)
            return visit_fields
        except Exception,e:
            logger.error("Process visit error:"+str(e))
            return None
    
    def process_source(self,source):
        """
        source=cookie_fields['__autmz']
        __autmz:area_hash.time_stamp.session_count.campagin_count.autmcsr=(autm_source)|autmccn=(autm_campaign)|autmcmd=(autm_medium)|autmctr=(autm_term)|autmcct=(autm_content);
        """
        if not source:
            return None
        try:
            source_fields={}
            area,timestamp,session,campagin,source=source.split('.',4)
            fields=source.split('|')
            for field in fields:
                m=re.match(r'([^=]+)=\(?([^\)]+)\)?',field)
                if not m:
                    continue
                key=m.group(1)
                value=m.group(2)
                if not source_fields.has_key(key):
                    source_fields[key]=value
            return source_fields
        except Exception,e:
            logger.error("Process source error:"+str(e))
            return None
        
    def process_url(self,url,is_decoded=True):
        """
        url=gif_fields['autmhn']+gif_fields['autmp']
        """
        return super(GaLineProcessor,self).process_url(url,is_decoded=is_decoded)
        
    def process_refer(self,refer,is_decoded=True,default_fields=[('refer_host',''),('refer_uri','/'),('refer_params','')]):
        """
        refer=gif_fields['autmr'] or gif_fields['refurl']
        """
        return super(GaLineProcessor,self).process_url(refer,is_decoded=is_decoded,default_fields=default_fields)
    
    def get_fields(self,line,to_list=True):
        try:
            line_fields=self.process_line(line)
            if line_fields is None:
                return None
            gif_fields=self.process_gif(line_fields['gif'])
            cookie_fields=self.process_cookie(gif_fields['autmcc'])
            visit_fields=self.process_visit(cookie_fields['__autma'],timestamp=line_fields['timestamp'])
            source_fields=self.process_source(cookie_fields['__autmz'])
            url_fields=self.process_url(gif_fields['autmhn']+gif_fields['autmp'])
            refer_fields=self.process_refer(gif_fields['autmr'] or gif_fields['refurl'])
            time_fields=self.process_time(line_fields['timestamp'])
            other_fields=self.process_other(other_func=None,fields=gif_fields)
            for fields in [line_fields,gif_fields,cookie_fields,visit_fields,source_fields,url_fields,refer_fields,time_fields,other_fields]:
                if not fields:
                    continue
                for key,value in fields.items():
                    if self.field_replacer.has_key(key):
                        key=self.field_replacer[key]
                    if self.field_dict.has_key(key):
                        if not self.fields.has_key(key):
                            self.fields[key]=value
            for field in self.field_dict.keys():
                if not self.fields.has_key(field):
                    self.fields[field]=self.field_dict[field][1]
            if to_list:
                return map(lambda key:self.fields[key],sorted(self.fields.keys(),key=lambda key:self.field_dict[key][0]))
            return self.fields
        except Exception,e:
            logger.error("Get fields error:"+str(e))
            return None
        
class ClickLineProcessor(LineProcessor):
    """
    LineProcessor for click access log
    """
    def __init__(self,**kwargs):
        super(ClickLineProcessor,self).__init__(**kwargs)
        self.line=self.define_line()
        self.gif=kwargs.get('gif',None)
        self.matcher=self.define_matcher()
        self.field_dict=self.define_field_dict()
        self.init_field_dict()
        self.field_replacer=self.define_field_replacer()
        self.dbc=DBC()
        self.dbi=DBI(**self.dbc.dbc('db_58_stat'))
        self.local_dict=self.define_local_dict()
        self.cate_dict=self.define_cate_dict()
        
    def define_line(self):
        return '''05eNEk+CajshcWikdKOlAg== 1362553201.779 183.15.13.87 - - [06/Mar/2013:15:00:01 +0800] GET /adsys/listclicklog?site_name=58&bi_val_pos=3&bi_enum_type=default&bi_enum_user=shangjia&bi_userid=5542016250119&post_id=12056972412164&rand=0.7286119984528255 HTTP/1.1 "200" 0 "http://sz.58.com/zufang/i2/pn2/key_%25E5%25A1%259E%25E7%25BA%25B3/?final=1&key=%E7%8E%89%E5%8D%8E%E8%8A%B1%E5%9B%AD" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)" "-" 0.000 - | -''';
        
    def define_matcher(self):
        if self.gif is None:
            self.gif='/adsys/listclicklog'
        matcher=r'^(?P<id58>[^\s]+)\s(?P<timestamp>[^\s]+)\s(?P<ip>[^\s]+)\s.*GET.*'+self.gif+'\?(?P<gif>[^\s]+)\sHTTP.*"(?P<url>http(?:[^"]+))"\s.*\|\s(?P<user_id>[\d-]+)$'  
        return matcher
    
    def define_field_dict(self):
        return {
            'log_type':(0,'click'),
            'id58':1,
            'click_timestamp':2,
            'click_time':3,
            'click_date':4,
            'click_year':5,
            'click_month':6,
            'click_day':7,
            'click_hour':8,
            'ip':9,
            'click_user':10,
            'site':11,
            'host':12,
            'uri':13,
            'params':14,
            'keyword':15,
            'is_search':17,
            'sort_type':18,
            'user_type':19,
            'post_user':20,
            'page_num':21,
            'page_pos':22,
            'product_type':23,
            'local_id':24,
            'local_name':25,
            'city1':26,
            'city2':27,
            'city3':28,
            'city4':29,
            'cate_id':30,
            'cate_name':31,
            'cate1':32,
            'cate2':33,
            'cate3':34,
            'cate4':35,
            'cate5':36,
            'post_id':37,
            'spread_url':38,
        }
    
    def define_field_replacer(self):
        return {
            'site_name':'site',
            'bi_userid':'post_user',
            'city':'local_id',
            'cate':'cate_id',
            'bi_enum_user':'user_type',
            'bi_enum_type':'product_type',
            'bi_val_pos':'page_pos',
            'timestamp':'click_timestamp',
            'time':'click_time',
            'date':'click_date',
            'year':'click_year',
            'month':'click_month',
            'day':'click_day',
            'hour':'click_hour',
            'user_id':'click_user',
        }
    def define_local_dict(self):
        try:
            local_dict={}
            self.dbi.connect()
            self.dbi.execute("select list_name,city1,city2,city3,city4 from display_local")
            for row in self.dbi.rows():
                local_dict[row[0]]=list(row[1:])
            self.dbi.close()
            return local_dict
        except Exception,e:
            logger.error('define local dict error: %s' % str(e))
            return None
    
    def define_cate_dict(self):
        try:
            cate_dict={}
            self.dbi.connect()
            self.dbi.execute("select list_name,cate1,cate2,cate3,cate4,cate5 from display_cate")
            for row in self.dbi.rows():
                cate_dict[row[0]]=list(row[1:])
            self.dbi.close()
            return cate_dict
        except Exception,e:
            logger.error('define cate dict error: %s' % str(e))
            return None
    
    def process_line(self,line,default_fields=['id58','timestamp','ip','gif','url','user_id']):
        try:        
            line_fields=super(ClickLineProcessor,self).process_line(line,default_fields=default_fields)
            click_user=line_fields.get('user_id','-1')
            click_user=-1 if click_user=='-' else click_user
            line_fields['user_id']=click_user
            return line_fields
        except Exception,e:
            logger.error('process line error: %s' % str(e))
            return None
        
    def process_gif(self,gif):
        try:
            gif_fields=super(ClickLineProcessor,self).process_gif(gif)
            post_id=gif_fields.get('post_id','-1')
            spread_url=''
            pm=re.match(r'^http://(?:[^/]+)(?:.*/(?P<post_id>[\d]+)x.shtml)?$',post_id)
            if pm:
                spread_url=post_id
                post_id=pm.group('post_id')
            elif not re.match(r'^\d+$',post_id):
                spread_url=post_id
                post_id='-1'
            gif_fields['post_id']=post_id
            gif_fields['spread_url']=spread_url
            gif_fields['city']=gif_fields.get('city','-1')
            gif_fields['cate']=gif_fields.get('cate','-1')
            return gif_fields
        except Exception,e:
            logger.error('process gif error: %s' % str(e))
            return None
        
    def process_url(self,url):
        try:
            url_fields=super(ClickLineProcessor,self).process_url(url,is_decoded=True)
            host,uri,params=url_fields['host'],url_fields['uri'],url_fields['params']
            local=host.split('.')[0]
            paths=[str(path) for path in uri.split('/') if path]
            page_num,keyword,lds,cds=0,'',[],[]
            for path in paths:
                pnm=re.match(r'^pn(?P<page_num>\d+)$',path)
                if pnm:
                    page_num=pnm.group('page_num')
                    continue
                kwm=re.match(r'(?:key|jh)_(?P<keyword>[^/]+)',path)
                if kwm:
                    keyword=kwm.group('keyword')
                    keyword=self.decode_str(keyword)
                    re.sub(r'\s+',' ',keyword)
                    re.sub(r'\n','',keyword)
                    continue
                if self.local_dict.has_key(path):
                    lds=[path]
                    lds.extend(self.local_dict[path])
                    continue
                if self.cate_dict.has_key(path):
                    cds=[path]
                    cds.extend(self.cate_dict[path])
                    continue
                
            url_fields['page_num']=page_num
            url_fields['keyword']=keyword
            url_fields['is_search']=1 if keyword else 0
            if len(lds)==0:
                lds=[local]
                if self.local_dict.has_key(local):
                    lds.extend(self.local_dict[local])
            for idx,name in enumerate(['local_name','city1','city2','city3','city4']):
                try:
                    city=lds[idx]
                except IndexError,e:
                    city=''
                url_fields[name]=city
            for idx,name in enumerate(['cate_name','cate1','cate2','cate3','cate4','cate5']):
                try:
                    cate=cds[idx]
                except IndexError,e:
                    cate=''
                url_fields[name]=cate
            sm=re.match(r'sort=(?P<sort_type>[^\&]+)',params)
            url_fields['sort_type']=sm.group('sort_type') if sm else ''
            return url_fields
        except Exception,e:
            logger.error('process url error: %s' % str(e))
            return None
    
    def get_fields(self,line,to_list=True):
        try:
            line_fields=self.process_line(line)
            if line_fields is None:
                return None
            gif_fields=self.process_gif(line_fields['gif'])
            url_fields=self.process_url(line_fields['url'])
            time_fields=self.process_time(line_fields['timestamp'])
            other_fields=self.process_other(other_func=None,fields=gif_fields)
            for fields in [line_fields,gif_fields,url_fields,time_fields,other_fields]:
                if not fields:
                    continue
                for key,value in fields.items():
                    if self.field_replacer.has_key(key):
                        key=self.field_replacer[key]
                    if self.field_dict.has_key(key):
                        if not self.fields.has_key(key):
                            self.fields[key]=value
            for field in self.field_dict.keys():
                if not self.fields.has_key(field):
                    self.fields[field]=self.field_dict[field][1]
            if to_list:
                return map(lambda key:self.fields[key],sorted(self.fields.keys(),key=lambda key:self.field_dict[key][0]))
            return self.fields
        except Exception,e:
            logger.error("Get fields error:"+str(e))
            return None
        
class ShowLineProcessor(LineProcessor):
    def __init__(self,**kwargs):
        super(ShowLineProcessor,self).__init__(**kwargs)
        self.line=self.define_line()
        self.gif=kwargs.get('gif',None)
        self.matcher=self.define_matcher()
        self.field_dict=self.define_field_dict()
        self.init_field_dict()
        self.field_replacer=self.define_field_replacer()
        
    def define_line(self):
        return '''05dvaFEtUmYaXhJxdHTwAg== 1362279552.170 60.252.240.93 - - [03/Mar/2013:10:59:12 +0800] GET /show/empty.js.gif?tag=pvlistshow&base_info=-1_375_1,12_11765208804103&info_data=j_1_13218861089031_12969960375938_3_1,j_1_7751597405447_10215011336451_3_1,j_1_5278905963015_12537617410696_3_1,j_1_10470674800903_11808031176324_3_1,j_1_10047204092166_13035596583939_3_1,j_1_10047036983047_13035631580933_3_1,j_1_10047036983047_13024512562306_3_1,j_1_10470674800903_11821032193671_3_1,j_1_10047036983047_12284375031562_3_1,j_1_10047174310663_11866340045698_3_1,j_1_10047099982342_12583474321155_3_1,j_1_11912635992326_12362024162312_3_1,j_1_10047138962439_13059838475266_3_1,j_1_10047099982342_11367953610120_3_1,j_1_10047099982342_11366988472708_3_1,j_1_10047267228423_12971381390849_3_1,j_1_10047234046983_12294600272261_3_1,j_1_10047036983047_12128432853635_3_1,j_1_6316425815047_12959216477569_3_1,j_1_10047267228423_13037829636234_3_1,j_1_8771971695110_12150411379976_3_1,j_1_8771971695110_12110093847688_3_1,j_1_13218933197319_13084785654534_3_1,j_1_13218933197319_13082240544265_3_1,j_1_13242673544966_13060459546889_3_1,j_1_11693579596039_12971981867523_3_1,j_1_13218907695879_13058111565318_3_1,j_1_13218883788295_13049957054083_3_1,j_1_10047204092166_13080568082690_3_1,j_1_12315998576134_12619054336390_3_1,j_1_56589994_12584254126345_3_1,j_1_56589994_12947699573765_3_1,j_1_12315998576134_12991682474881_3_1,j_1_10047267228423_12725522796810_3_1,j_1_10047099982342_11368990711171_3_1&datastamp=1362279551015 HTTP/1.1 "200" 35 "http://dq.58.com/ershoufang/?key=%E9%93%B6%E4%BA%BFa%E5%8C%BA&cmcskey=%E9%93%B6%E4%BA%BFa%E5%8C%BA&final=1&specialtype=gls" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; KB974488)" "-" 0.000 - | 11765208804103'''        
    
    def define_matcher(self):
        if self.gif is None:
            self.gif='/show/empty.js.gif'
        matcher=r'^(?P<id58>[^\s]+)\s(?P<timestamp>[^\s]+)\s(?P<ip>[^\s]+)\s.*GET.*'+self.gif+'\?(?P<gif>[^\s]+)\sHTTP.*"(?P<url>http(?:[^"]+))"\s.*\|\s(?P<user_id>[\d-]+)$'  
        return matcher
    
    def define_field_dict(self):
        return {
            'log_type':(0,'show'),
            'id58':1,
            'show_timestamp':2,
            'show_time':3,
            'show_date':4,
            'show_year':5,
            'show_month':6,
            'show_day':7,
            'show_hour':8,
            'ip':9,
            'user_id':10,
            'host':11,
            'uri':12,
            'params':13,
            'page_tag':14,
            'page_info':15,
            'post_info':16,
        }
    
    def define_field_replacer(self):
        return {
            'tag':'page_tag',
            'base_info':'page_info',
            'info_data':'post_info',
            'timestamp':'show_timestamp',
            'time':'show_time',
            'date':'show_date',
            'year':'show_year',
            'month':'show_month',
            'day':'show_day',
            'hour':'show_hour',
        }
        
    def process_line(self,line,default_fields=['id58','timestamp','ip','gif','url','user_id']):
        try:        
            line_fields=super(ShowLineProcessor,self).process_line(line,default_fields=default_fields)
            user_id=line_fields.get('user_id','-1')
            user_id=-1 if user_id=='-' else user_id
            line_fields['user_id']=user_id
            return line_fields
        except Exception,e:
            logger.error('process line error: %s' % str(e))
            return None
    
    def get_fields(self,line,to_list=True):
        try:
            line_fields=self.process_line(line)
            if line_fields is None:
                return None
            gif_fields=self.process_gif(line_fields['gif'])
            url_fields=self.process_url(line_fields['url'])
            time_fields=self.process_time(line_fields['timestamp'])
            other_fields=self.process_other(other_func=None,fields=gif_fields)
            for fields in [line_fields,gif_fields,url_fields,time_fields,other_fields]:
                if not fields:
                    continue
                for key,value in fields.items():
                    if self.field_replacer.has_key(key):
                        key=self.field_replacer[key]
                    if self.field_dict.has_key(key):
                        if not self.fields.has_key(key):
                            self.fields[key]=value
            for field in self.field_dict.keys():
                if not self.fields.has_key(field):
                    self.fields[field]=self.field_dict[field][1]
            if to_list:
                return map(lambda key:self.fields[key],sorted(self.fields.keys(),key=lambda key:self.field_dict[key][0]))
            return self.fields
        except Exception,e:
            logger.error("Get fields error:"+str(e))
            return None
    
if __name__ == '__main__':
    lp=GaLineProcessor()
    #print "line=%s" % lp.line
    lp.line='''123.150.182.169 - - [15/Jan/2013:00:03:28 +0800] "GET /log/caipiao/collectlog.gif?&autmn=1618366689&autmhn=caipiao.58.com&autmcs=UTF-8&autmsr=800x600&autmsc=24-bit&autmul=zh-cn&autmje=0&autmfl=10.0%20r22&autmhid=629286436&autmr=http%3A%2F%2Ftj.58.com%2F&autmp=%2F%3Ffrom%3Dhome-hotkey-caipiao&autmcc=__autma%3D253535702.400542011.1358179409.1358179409.1358179409.1%3B%2B__autmz%3D253535702.1358179409.1.1.autmcsr%3Dtj.58.com%7Cautmccn%3D(referral)%7Cautmcmd%3Dreferral%7Cautmcct%3D%2F%3B&id58=05dvaFD0K74EiQ9ayeIbAg==&refurl=http://tj.58.com/?&test=abc HTTP/1.1" 200 0 "http://caipiao.58.com/?from=home-hotkey-caipiao" "-" "-" 1358179408.863'''
    fields=lp.get_fields(line=lp.line,to_list=False)
    print "fields=%s" % fields
    #lp=ClickLineProcessor()
    #print "line=%s" % lp.line
    #fields=lp.get_fields(line=lp.line,to_list=True)
    #print sorted(lp.fields.keys(),key=lambda key:lp.field_dict[key][0])
    #print "fields=%s" % fields
    #lp=ShowLineProcessor()
    ##print "line=%s" % lp.line
    #fields=lp.get_fields(line=lp.line,to_list=False)
    #print "fields=%s" % fields
    
    
